import soundfile as sf
import torch
from modelscope import Qwen2_5OmniModel, Qwen2_5OmniProcessor
from qwen_omni_utils import process_mm_info

# default: Load the model on the available device(s)
# model = Qwen2_5OmniModel.from_pretrained("/root/autodl-tmp/Qwen/Qwen2.5-Omni-7B", torch_dtype="auto", device_map="auto")

# We recommend enabling flash_attention_2 for better acceleration and memory saving.
model = Qwen2_5OmniModel.from_pretrained(
    "/root/autodl-tmp/Qwen/Qwen2.5-Omni-7B",
    torch_dtype=torch.bfloat16,
    device_map="cuda",
    attn_implementation="flash_attention_2",
    weights_only=True, 
)

processor = Qwen2_5OmniProcessor.from_pretrained("/root/autodl-tmp/Qwen/Qwen2.5-Omni-7B",use_fast= True)

# 单视频
conversation = [
    {
        "role": "system",
        "content": "You are Qwen, a virtual human developed by the Qwen Team, Alibaba Group, capable of perceiving auditory and visual inputs, as well as generating text and speech.",
    },
    {
        "role": "user",
        "content": [
            {"type": "video", "video": "tmp/划船.avi"},
            {"type": "text", "text": "please tell me about this videos"}
        ],
    },
]


# 多视频
conversation = [
    {
        "role": "system",
        "content": "You are Qwen, a virtual human developed by the Qwen Team, Alibaba Group, capable of perceiving auditory and visual inputs, as well as generating text and speech.",
    },
    {
        "role": "user",
        "content": [
            {"type": "video", "video": "tmp/划船.avi"},
            {"type": "video", "video": "tmp/花滑.avi"},
            {"type": "text", "text": "please tell me  the differences about these videos"}
        ],
    },
]

# 单图
conversation = [
    {
        "role": "system",
        "content": "You are Qwen, a virtual human developed by the Qwen Team, Alibaba Group, capable of perceiving auditory and visual inputs, as well as generating text and speech.",
    },
    {
        "role": "user",
        "content": [
            {"type": "image", "image": "tmp/吃饭1.png"},
            
            {"type": "text", "text": "please tell me about the image"}
        ],
    },
]

# 多图
conversation = [
    {
        "role": "system",
        "content": "You are Qwen, a virtual human developed by the Qwen Team, Alibaba Group, capable of perceiving auditory and visual inputs, as well as generating text and speech.",
    },
    {
        "role": "user",
        "content": [
            {"type": "image", "image": "tmp/吃饭1.png"},
            {"type": "image", "image": "tmp/上课1.png"},
            {"type": "text", "text": "please tell me the differences about these images"}
        ],
    },
]

# n图 n视频
conversation = [
    {
        "role": "system",
        "content": "You are Qwen, a virtual human developed by the Qwen Team, Alibaba Group, capable of perceiving auditory and visual inputs, as well as generating text and speech.",
    },
    {
        "role": "user",
        "content": [
            {"type": "image", "image": "tmp/吃饭1.png"},
            {"type": "image", "image": "tmp/上课1.png"},
            {"type": "video", "video": "tmp/划船.avi"},
            {"type": "video", "video": "tmp/花滑.avi"},
            {"type": "text", "text": "please tell me the differences about these images and videos"}
        ],
    },
]

# 单音频
conversation = [
    {
        "role": "system",
        "content": "You are Qwen, a virtual human developed by the Qwen Team, Alibaba Group, capable of perceiving auditory and visual inputs, as well as generating text and speech.",
    },
    {
        "role": "user",
        "content": [
            {"type": "audio", "audio": "tmp/录音1.wav"},
            {"type": "text", "text": "please tell me the audio"}
        ],
    },
]

# 多音频（英语 or 中文）
conversation = [
    {
        "role": "system",
        "content": "You are Qwen, a virtual human developed by the Qwen Team, Alibaba Group, capable of perceiving auditory and visual inputs, as well as generating text and speech.",
    },
    {
        "role": "user",
        "content": [
            {"type": "audio", "audio": "tmp/录音1.wav"},
            {"type": "audio", "audio": "tmp/录音3.wav"},
            {"type": "text", "text": "What are the elements can you  hear in these medias?"}
        ],
    },
]

# 视频中提问（英语 or 中文）
conversation = [
    {
        "role": "system",
        "content": "You are Qwen, a virtual human developed by the Qwen Team, Alibaba Group, capable of perceiving auditory and visual inputs, as well as generating text and speech.",
    },
    {
        "role": "user",
        "content": [
            {"type": "audio", "audio": "tmp/录屏1.mp4"},
            {"type": "text", "text": "tell me the answer about the video"}
        ],
    },
]

# # 全模态（英语 or 中文）
# conversation = [
#     {
#         "role": "system",
#         "content": "You are Qwen, a virtual human developed by the Qwen Team, Alibaba Group, capable of perceiving auditory and visual inputs, as well as generating text and speech.",
#     },
#     {
#         "role": "user",
#         "content": [
#             {"type": "image", "image": "/path/to/image.jpg"},
#             {"type": "video", "video": "/path/to/video.mp4"},
#             {"type": "audio", "audio": "/path/to/audio.wav"},
#             {"type": "text", "text": "What are the elements can you see and hear in these medias?"},
#         ],
#     },
# ]






# Preparation for inference
text = processor.apply_chat_template(conversation, add_generation_prompt=True, tokenize=False)
audios, images, videos = process_mm_info(conversation, use_audio_in_video=True)
inputs = processor(text=text, audios=audios, images=images, videos=videos, return_tensors="pt", padding=True)
inputs = inputs.to(model.device).to(model.dtype)

# Inference: Generation of the output text and audio
text_ids, audio = model.generate(**inputs, use_audio_in_video=True,spk="Ethan")

text = processor.batch_decode(text_ids, skip_special_tokens=True, clean_up_tokenization_spaces=False)
print(text)
sf.write(
    "output.wav",
    audio.reshape(-1).detach().cpu().numpy(),
    samplerate=24000,
)